Libraries

#others
import pandas as pd
import numpy as np

#retrieving data
import requests
import json

#plotting
import plotly.express as px
import plotly.graph_objects as go
from ipywidgets import interact, fixed, IntSlider
import matplotlib.pyplot as plt

#optimization
from scipy.optimize import minimize

Retrieve data

#downloading data from coingecko.com
def download_historical_data(crypto_id, days):
    url = f'https://api.coingecko.com/api/v3/coins/{crypto_id}/market_chart'

    # Set the number of days for historical data
    params = {
        'vs_currency': 'usd',
        'days': days
    }

    headers = {
        'Content-Type': 'application/json'
    }

    try:
        response = requests.get(url, params=params, headers=headers)
        data = response.json()

        # Save the data to a json file
        filename = f'{crypto_id}_historical_data.json'
        with open(filename, 'w') as f:
            json.dump(data, f)

        print(f"Downloaded historical data for {crypto_id} successfully and saved to {filename}!")

    except requests.exceptions.RequestException as e:
        print(f"Error occurred while downloading data for {crypto_id}: {e}")

# Specify the crypto IDs and the number of days of historical data you want to download
crypto_ids = ['bitcoin', 'ethereum', 'litecoin', 'dogecoin', 'vechain', 'filecoin', 'binancecoin', 'cardano', 'ripple', 'polkadot']
days = 365

# Iterate through the list of crypto IDs and download the data
for crypto_id in crypto_ids:
    download_historical_data(crypto_id, days)
    
dfs = []  # List to store individual DataFrames

for crypto_id in crypto_ids:
    # Load the JSON file
    filename = f'{crypto_id}_historical_data.json'
    with open(filename, 'r') as f:
        data = json.load(f)

    # Extract the price data from the JSON structure
    prices = data['prices']

    # Create a Pandas DataFrame
    df = pd.DataFrame(prices, columns=['Timestamp', f'{crypto_id}_Price'])

    # Convert the timestamp to datetime
    df['Timestamp'] = pd.to_datetime(df['Timestamp'], unit='ms')

    # Round down the timestamp to the nearest day
    df['Timestamp'] = df['Timestamp'].dt.floor('D')

    # Group by timestamp and take the mean of prices
    df = df.groupby('Timestamp').mean()

    # Add the DataFrame to the list
    dfs.append(df)

# Merge all the DataFrames into a single DataFrame based on the Timestamp index
indices = pd.concat(dfs, axis=1)

# Print the merged DataFrame
indices.head()

Downloaded historical data for bitcoin successfully and saved to bitcoin_historical_data.json!
Downloaded historical data for ethereum successfully and saved to ethereum_historical_data.json!
Downloaded historical data for litecoin successfully and saved to litecoin_historical_data.json!
Downloaded historical data for dogecoin successfully and saved to dogecoin_historical_data.json!
Downloaded historical data for vechain successfully and saved to vechain_historical_data.json!
Downloaded historical data for filecoin successfully and saved to filecoin_historical_data.json!
Downloaded historical data for binancecoin successfully and saved to binancecoin_historical_data.json!
Downloaded historical data for cardano successfully and saved to cardano_historical_data.json!
Downloaded historical data for ripple successfully and saved to ripple_historical_data.json!
Downloaded historical data for polkadot successfully and saved to polkadot_historical_data.json!

	bitcoin_Price	ethereum_Price	litecoin_Price	dogecoin_Price	vechain_Price	filecoin_Price	binancecoin_Price	cardano_Price	ripple_Price	polkadot_Price
Timestamp
2022-06-11	29101.298260	1663.844367	56.691766	0.075630	0.029500	6.823364	286.081231	0.574789	0.382066	8.656939
2022-06-12	28374.144997	1530.038617	52.114902	0.069771	0.027636	6.094975	269.864619	0.554015	0.360067	8.007774
2022-06-13	26767.269173	1454.686760	47.984326	0.064155	0.025897	5.717548	256.082808	0.502399	0.347372	7.450028
2022-06-14	22525.768350	1205.595286	43.291987	0.054117	0.023805	5.335227	222.637752	0.461422	0.311383	7.001811
2022-06-15	22244.848968	1214.866265	46.153964	0.055625	0.024064	5.491086	223.454259	0.485459	0.322955	7.379033

Pre-processing

indices.isna().sum()
#There is no missing in dataset

bitcoin_Price        0
ethereum_Price       0
litecoin_Price       0
dogecoin_Price       0
vechain_Price        0
filecoin_Price       0
binancecoin_Price    0
cardano_Price        0
ripple_Price         0
polkadot_Price       0
dtype: int64

# Handle outliers by winsorizing at 5% and 95% percentiles
pct_low, pct_high = 0.05, 0.95
indices = indices.apply(lambda x: np.clip(x, x.quantile(pct_low), x.quantile(pct_high)))

#Calculation of the daily and cumulative returns for each asset
returns = indices.pct_change().dropna()
cum_returns = returns.add(1).cumprod().sub(1).dropna()*100

#plotting Cumulative returns
fig = px.line(cum_returns, x=cum_returns.index, y=cum_returns.columns, title='Cumulative Returns of Indices 1Y')
fig.update_xaxes(title_text='Date')
fig.update_yaxes(title_text='Cumulative Return in %')
fig.show()

#Calculation rolling volatility for each asset
window_size = 7  # Specify the window size for the rolling calculation
volatility = returns.rolling(window=window_size).std()

#plotting volatility
fig = px.line(volatility, x=cum_returns.index, y=cum_returns.columns, title='Volatility of Indices 1Y')
fig.update_xaxes(title_text='Date')
fig.update_yaxes(title_text='Volatility')
fig.show()

Implementing models and evaluation

def maximum_sharpe_ratio_portfolio(returns, allow_shorting=True):
    num_assets = returns.shape[1]
    
    # Define the objective function for maximum Sharpe ratio portfolio
    def objective_function(weights):
        # Calculate portfolio mean return
        mean_returns = returns.mean()        
        # Calculate portfolio variance-covariance matrix
        covariance_matrix = returns.cov()
        # Calculate portfolio return and variance
        portfolio_mean = np.sum(mean_returns*weights)
        portfolio_std = np.sqrt(np.dot(weights.T, np.dot(covariance_matrix, weights)))        
        # Calculate ratio
        sharpe_ratio = portfolio_mean / portfolio_std
        return -sharpe_ratio
    
    # Define the constraints
    constraints = ({'type': 'eq', 'fun': lambda weights: np.sum(weights) - 1.0})
    
    if allow_shorting:
        bounds = [(None, None)] * num_assets  # No bounds on weights
    else:
        bounds = [(0, None)] * num_assets  # No shorting allowed
    
    # Perform the optimization
    initial_weights = np.ones(num_assets) / num_assets  # Start with equal weights
    optimized_weights = minimize(objective_function, initial_weights, bounds=bounds, constraints=constraints, method='SLSQP').x
    return optimized_weights

def minimum_variance_portfolio(returns, allow_shorting=True):
    num_assets = returns.shape[1]

    def objective_function(weights):
        mean_returns = returns.mean()
        covariance_matrix = returns.cov()
        portfolio_mean = np.sum(mean_returns*weights)
        portfolio_variance = np.sqrt(np.dot(weights.T, np.dot(covariance_matrix, weights)))
        return portfolio_variance
    
    constraints = ({'type': 'eq', 'fun': lambda weights: np.sum(weights) - 1.0})
    
    if allow_shorting:
        bounds = [(None, None)] * num_assets
    else:
        bounds = [(0, None)] * num_assets
    
    initial_weights = np.ones(num_assets) / num_assets
    optimized_weights = minimize(objective_function, initial_weights, bounds=bounds, constraints=constraints, method='SLSQP').x
    return optimized_weights


# Portfolio Performance Evaluation
def evaluate_portfolio(returns, weights, rebalancing_frequency='daily'):
    n = len(returns)
    if rebalancing_frequency == 'daily':
        returns_rebalanced = returns
    elif rebalancing_frequency == 'weekly':
        returns_rebalanced = returns[::7]
    elif rebalancing_frequency == 'monthly':
        returns_rebalanced = returns[::30]
    else:
        raise ValueError("Invalid rebalancing frequency. Choose from 'daily', 'weekly', or 'monthly'.")
    
    portfolio_returns = np.dot(returns_rebalanced, weights)
    portfolio_cumulative_returns = np.cumprod(portfolio_returns + 1) - 1
    
    portfolio_stats = {
        'Expected Return': np.mean(portfolio_returns),
        'Volatility': np.std(portfolio_returns),
        'Sharpe Ratio': np.mean(portfolio_returns) / np.std(portfolio_returns),
        'Cumulative Returns': portfolio_cumulative_returns[-1]
    }
    
    return portfolio_stats, portfolio_cumulative_returns

# Compute daily, weekly, and monthly rebalanced portfolios
rebalancing_rules = ['daily', 'weekly', 'monthly']
portfolio_optimization_models = [minimum_variance_portfolio, maximum_sharpe_ratio_portfolio]

for rebalancing_rule in rebalancing_rules:
    for optimization_model in portfolio_optimization_models:
        # Calculate portfolio weights
        weights = optimization_model(returns, allow_shorting=False)
        # Evaluate portfolio performance
        stats, cumulative_returns = evaluate_portfolio(returns, weights, rebalancing_frequency=rebalancing_rule)
        # Calculate asset allocation
        allocation = pd.DataFrame(np.round(weights*100,4),index=returns.columns,columns=['allocation'])
        allocation['allocation'] = allocation['allocation'].map('{:.1f}%'.format)
        # Print portfolio statistics
        print(f"Rebalancing rule: {rebalancing_rule}")
        print(f"Optimization model: {optimization_model.__name__}")
        print(f"Assets:")
        print(allocation.to_string(header=False))
        print(f"Expected returns: {round(stats['Expected Return'],4)}")
        print(f"Cumulative returns: {round(stats['Cumulative Returns'],4)}")
        print(f"Volatility: {round(stats['Volatility'],4)}")
        print(f"Sharpe ratio: {round(stats['Sharpe Ratio'],4)}")
       # Plot cumulative returns
        if rebalancing_rule == 'daily':
            fig = px.line(cumulative_returns, x=cum_returns.index, y=cumulative_returns, title='Cumulative Returns of Indices 1Y')
        elif rebalancing_rule == 'weekly':
            fig = px.line(cumulative_returns, x=cum_returns.index[::7], y=cumulative_returns, title='Cumulative Returns of Indices 1Y')
        elif rebalancing_rule == 'monthly':
            fig = px.line(cumulative_returns, x=cum_returns.index[::30], y=cumulative_returns, title='Cumulative Returns of Indices 1Y')
        else:
            raise ValueError("Invalid rebalancing frequency. Choose from 'daily', 'weekly', or 'monthly'.")
        fig.update_xaxes(title_text='Date')
        fig.update_yaxes(title_text='Cumulative Return in %')
        fig.show()
        print("-"*80)

Rebalancing rule: daily
Optimization model: minimum_variance_portfolio
Assets:
bitcoin_Price      45.5%
ethereum_Price      0.0%
litecoin_Price      4.3%
dogecoin_Price      0.0%
vechain_Price       0.0%
filecoin_Price      0.0%
binancecoin_Price  42.5%
cardano_Price       0.0%
ripple_Price        7.8%
polkadot_Price      0.0%
Expected returns: 0.0003
Cumulative returns: -0.0271
Volatility: 0.0257
Sharpe ratio: 0.0101

--------------------------------------------------------------------------------
Rebalancing rule: daily
Optimization model: maximum_sharpe_ratio_portfolio
Assets:
bitcoin_Price       0.0%
ethereum_Price      0.0%
litecoin_Price     62.8%
dogecoin_Price      0.0%
vechain_Price       0.0%
filecoin_Price      0.0%
binancecoin_Price   0.0%
cardano_Price       0.0%
ripple_Price       37.2%
polkadot_Price      0.0%
Expected returns: 0.0016
Cumulative returns: 0.498
Volatility: 0.0325
Sharpe ratio: 0.0503

--------------------------------------------------------------------------------
Rebalancing rule: weekly
Optimization model: minimum_variance_portfolio
Assets:
bitcoin_Price      45.5%
ethereum_Price      0.0%
litecoin_Price      4.3%
dogecoin_Price      0.0%
vechain_Price       0.0%
filecoin_Price      0.0%
binancecoin_Price  42.5%
cardano_Price       0.0%
ripple_Price        7.8%
polkadot_Price      0.0%
Expected returns: 0.0006
Cumulative returns: 0.0257
Volatility: 0.0143
Sharpe ratio: 0.0413

--------------------------------------------------------------------------------
Rebalancing rule: weekly
Optimization model: maximum_sharpe_ratio_portfolio
Assets:
bitcoin_Price       0.0%
ethereum_Price      0.0%
litecoin_Price     62.8%
dogecoin_Price      0.0%
vechain_Price       0.0%
filecoin_Price      0.0%
binancecoin_Price   0.0%
cardano_Price       0.0%
ripple_Price       37.2%
polkadot_Price      0.0%
Expected returns: 0.0009
Cumulative returns: 0.033
Volatility: 0.0214
Sharpe ratio: 0.04

--------------------------------------------------------------------------------
Rebalancing rule: monthly
Optimization model: minimum_variance_portfolio
Assets:
bitcoin_Price      45.5%
ethereum_Price      0.0%
litecoin_Price      4.3%
dogecoin_Price      0.0%
vechain_Price       0.0%
filecoin_Price      0.0%
binancecoin_Price  42.5%
cardano_Price       0.0%
ripple_Price        7.8%
polkadot_Price      0.0%
Expected returns: -0.0
Cumulative returns: -0.0071
Volatility: 0.0328
Sharpe ratio: -0.0005

--------------------------------------------------------------------------------
Rebalancing rule: monthly
Optimization model: maximum_sharpe_ratio_portfolio
Assets:
bitcoin_Price       0.0%
ethereum_Price      0.0%
litecoin_Price     62.8%
dogecoin_Price      0.0%
vechain_Price       0.0%
filecoin_Price      0.0%
binancecoin_Price   0.0%
cardano_Price       0.0%
ripple_Price       37.2%
polkadot_Price      0.0%
Expected returns: -0.0071
Cumulative returns: -0.1028
Volatility: 0.0486
Sharpe ratio: -0.1451

--------------------------------------------------------------------------------

def portfolio_performance(weights, mean_returns, cov_matrix):
    """
    Calculates the portfolio's standard deviation and annualized return.
    
    Args:
        weights (np.ndarray): Array of portfolio weights.
        mean_returns (pd.Series): Series of mean returns for each asset.
        cov_matrix (pd.DataFrame): Covariance matrix of asset returns.
    
    Returns:
        Tuple: Standard deviation and annualized return of the portfolio.
    """
    returns = np.sum(mean_returns*weights)*365
    std = np.sqrt(np.dot(weights.T, np.dot(cov_matrix, weights)))*np.sqrt(365)
    return std, returns

def random_portfolios(num_portfolios, mean_returns, cov_matrix):
    """
    Generates random portfolios with given number of portfolios and asset information.
    
    Args:
        num_portfolios (int): Number of random portfolios to generate.
        mean_returns (pd.Series): Series of mean returns for each asset.
        cov_matrix (pd.DataFrame): Covariance matrix of asset returns.
    
    Returns:
        Tuple: Array of portfolio standard deviations, returns, and Sharpe ratios, and a list of portfolio weights.
    """
    results = np.zeros((3,num_portfolios))
    weights_record = []
    
    num_assets = len(mean_returns)
    
    for i in range(num_portfolios):
        weights = np.random.random(num_assets)
        weights /= np.sum(weights)
        weights_record.append(weights)
        portfolio_std_dev, portfolio_return = portfolio_performance(weights, mean_returns, cov_matrix)
        results[0,i] = portfolio_std_dev
        results[1,i] = portfolio_return
        results[2,i] = portfolio_return / portfolio_std_dev
    
    return results, weights_record

def efficient_return(mean_returns, cov_matrix, target):
    """
    Finds the weights for an efficient portfolio given a target return.
    
    Args:
        mean_returns (pd.Series): Series of mean returns for each asset.
        cov_matrix (pd.DataFrame): Covariance matrix of asset returns.
        target (float): Target portfolio return.
    
    Returns:
        scipy.optimize.OptimizeResult: Result of the optimization with the weights for the efficient portfolio.
    """
    num_assets = len(mean_returns)
    args = (mean_returns, cov_matrix)

    def portfolio_return(weights):
        return portfolio_performance(weights, mean_returns, cov_matrix)[1]
    
    def portfolio_volatility(weights, mean_returns, cov_matrix):
        return portfolio_performance(weights, mean_returns, cov_matrix)[0]

    constraints = ({'type': 'eq', 'fun': lambda x: portfolio_return(x) - target},
                   {'type': 'eq', 'fun': lambda x: np.sum(x) - 1})
    bounds = tuple((0,1) for asset in range(num_assets))
    result = minimize(portfolio_volatility, num_assets*[1./num_assets,], args=args, method='SLSQP', bounds=bounds, constraints=constraints)
    return result

def efficient_frontier(mean_returns, cov_matrix, returns_range):
    """
    Generates a list of efficient portfolios along the efficient frontier.
    
    Args:
        mean_returns (pd.Series): Series of mean returns for each asset.
        cov_matrix (pd.DataFrame): Covariance matrix of asset returns.
        returns_range (np.ndarray): Array of target returns.
    
    Returns:
        List: List of OptimizeResult objects representing the efficient portfolios.
    """
    efficients = []
    for ret in returns_range:
        efficients.append(efficient_return(mean_returns, cov_matrix, ret))
    return efficients

def display_calculated_ef_with_random(returns, portfolio_optimization_models, num_portfolios=10000, interactive=False):
    num_assets = returns.shape[1]
    # Calculate portfolio mean return
    mean_returns = returns.mean()
    # Calculate portfolio variance-covariance matrix
    covariance_matrix = returns.cov()
    def plot_efficient_frontier(num_portfolios):
        # Generate random portfolios
        results, _ = random_portfolios(num_portfolios, mean_returns, covariance_matrix)
        results=pd.DataFrame(results).T
        results = results.rename(columns={2: 'Sharpe Ratio'})
        # Plot scatter plot of random portfolios
        fig = px.scatter(results, x=0, y=1, color='Sharpe Ratio', color_continuous_scale='YlGnBu', opacity=0.3,
                         labels={0: 'Volatility', 1: 'Returns'})
        # Plot optimized portfolios
        for optimization_model in portfolio_optimization_models:
            weights = optimization_model(returns, allow_shorting=False)
            sdp, rp = portfolio_performance(weights, mean_returns, covariance_matrix)
            fig.add_trace(go.Scatter(x=[sdp], y=[rp], mode='markers', name=optimization_model.__name__))
        # Plot efficient frontier
        target = np.linspace(0, 0.65, 50)
        efficient_portfolios = efficient_frontier(mean_returns, covariance_matrix, target)
        fig.add_trace(go.Scatter(x=[p['fun'] for p in efficient_portfolios], y=target,
                                 mode='lines', line=dict(dash='dash'), name='Efficient Frontier'))
        # Update layout and display the plot
        fig.update_layout(title='Calculated Portfolio Optimization based on Efficient Frontier',
                          xaxis_title='Volatility (annualized)', yaxis_title='Returns (annualized)', showlegend=True)
        fig.update_layout(legend=dict(x=0, y=1, traceorder='normal', font=dict(size=12), bgcolor='LightSteelBlue', bordercolor='Black', borderwidth=2))
        fig.show()
    # Create interactive widget
    if interactive:
        interact(plot_efficient_frontier, num_portfolios=IntSlider(min=10000, max=100000, step=10000, continuous_update=False))
    else:
        return plot_efficient_frontier(num_portfolios)

display_calculated_ef_with_random(returns, portfolio_optimization_models, num_portfolios=100000)

display_calculated_ef_with_random(returns, portfolio_optimization_models,  num_portfolios=10000, interactive=True)

interactive(children=(IntSlider(value=10000, continuous_update=False, description='num_portfolios', max=100000&

Q-learning algorithm for automated stock trading - additional part

from collections import deque
import random
import tensorflow.compat.v1 as tf
tf.compat.v1.disable_eager_execution()

class Agent:
    def __init__(self, state_size, window_size, trend, skip, batch_size):
        # Initialize the Agent object with necessary attributes
        self.state_size = state_size
        self.window_size = window_size
        self.half_window = window_size // 2
        self.trend = trend
        self.skip = skip
        self.action_size = 3
        self.batch_size = batch_size
        self.memory = deque(maxlen=1000)
        self.inventory = []
        self.gamma = 0.95
        self.epsilon = 0.5
        self.epsilon_min = 0.01
        self.epsilon_decay = 0.999
        
        # TensorFlow graph setup
        tf.reset_default_graph()
        self.sess = tf.InteractiveSession()
        self.X = tf.placeholder(tf.float32, [None, self.state_size])
        self.Y = tf.placeholder(tf.float32, [None, self.action_size])
        feed = tf.layers.dense(self.X, 256, activation=tf.nn.relu)
        self.logits = tf.layers.dense(feed, self.action_size)
        self.cost = tf.reduce_mean(tf.square(self.Y - self.logits))
        self.optimizer = tf.train.GradientDescentOptimizer(1e-5).minimize(self.cost)
        self.sess.run(tf.global_variables_initializer())
        
    def act(self, state):
        # Choose an action based on the current state
        if random.random() <= self.epsilon:
            return random.randrange(self.action_size)
        return np.argmax(self.sess.run(self.logits, feed_dict={self.X: state})[0])
    
    def get_state(self, t):
        # Generate the state representation for a given time step
        window_size = self.window_size + 1
        d = t - window_size + 1
        block = self.trend[d : t + 1] if d >= 0 else -d * [self.trend[0]] + self.trend[0 : t + 1]
        res = []
        for i in range(window_size - 1):
            res.append(block[i + 1] - block[i])
        return np.array([res])
    
    def replay(self, batch_size):
        # Replay memory to train the agent
        mini_batch = []
        l = len(self.memory)
        for i in range(l - batch_size, l):
            mini_batch.append(self.memory[i])
        replay_size = len(mini_batch)
        X = np.empty((replay_size, self.state_size))
        Y = np.empty((replay_size, self.action_size))
        states = np.array([a[0][0] for a in mini_batch])
        new_states = np.array([a[3][0] for a in mini_batch])
        Q = self.sess.run(self.logits, feed_dict={self.X: states})
        Q_new = self.sess.run(self.logits, feed_dict={self.X: new_states})
        for i in range(len(mini_batch)):
            state, action, reward, next_state, done = mini_batch[i]
            target = Q[i]
            target[action] = reward
            if not done:
                target[action] += self.gamma * np.amax(Q_new[i])
            X[i] = state
            Y[i] = target
        cost, _ = self.sess.run([self.cost, self.optimizer], feed_dict={self.X: X, self.Y: Y})
        if self.epsilon > self.epsilon_min:
            self.epsilon *= self.epsilon_decay
        return cost
    
    def buy(self, initial_money):
        # Buy/sell actions based on the current state and available money
        starting_money = initial_money
        states_sell = []
        states_buy = []
        inventory = []
        state = self.get_state(0)
        for t in range(0, len(self.trend) - 1, self.skip):
            action = self.act(state)
            next_state = self.get_state(t + 1)
            if action == 1 and initial_money >= self.trend[t] and t < (len(self.trend) - self.half_window):
                inventory.append(self.trend[t])
                initial_money -= self.trend[t]
                states_buy.append(t)
                print('day %d: buy 1 unit at price %f, total balance %f' % (t, self.trend[t], initial_money))
            elif action == 2 and len(inventory):
                bought_price = inventory.pop(0)
                initial_money += self.trend[t]
                states_sell.append(t)
                try:
                    invest = ((self.trend[t] - bought_price) / bought_price) * 100
                except:
                    invest = 0
                print(
                    'day %d, sell 1 unit at price %f, total balance %f'
                    % (t, self.trend[t], initial_money)
                )
            state = next_state
        invest = ((initial_money - starting_money) / starting_money) * 100
        total_gains = initial_money - starting_money
        return states_buy, states_sell, total_gains, invest
    
    def train(self, iterations, checkpoint, initial_money):
        # Train the agent for a certain number of iterations
        for i in range(iterations):
            total_profit = 0
            inventory = []
            state = self.get_state(0)
            starting_money = initial_money
            for t in range(0, len(self.trend) - 1, self.skip):
                action = self.act(state)
                next_state = self.get_state(t + 1)
                if action == 1 and starting_money >= self.trend[t] and t < (len(self.trend) - self.half_window):
                    inventory.append(self.trend[t])
                    starting_money -= self.trend[t]
                elif action == 2 and len(inventory) > 0:
                    bought_price = inventory.pop(0)
                    total_profit += self.trend[t] - bought_price
                    starting_money += self.trend[t]
                invest = ((starting_money - initial_money) / initial_money)
                self.memory.append(
                    (state, action, invest, next_state, starting_money < initial_money)
                )
                state = next_state
                batch_size = min(self.batch_size, len(self.memory))
                cost = self.replay(batch_size)
            if (i + 1) % checkpoint == 0:
                print(
                    'epoch: %d, reward: %f.3, loss: %f'
                    % (i + 1, total_profit, cost)
                )

chosen_crypto = ['litecoin_Price', 'vechain_Price', 'filecoin_Price']

#initialization
initial_money = 100000
window_size = 30
skip = 1
batch_size = 32

for crypto_id in chosen_crypto:
    #df of crypto chosen
    df = indices[crypto_id].values.tolist()
    #defining agent
    agent = Agent(state_size = window_size, 
              window_size = window_size, 
              trend = df, 
              skip = skip, 
              batch_size = batch_size)
    #training
    agent.train(iterations = 100, checkpoint = 10, initial_money = initial_money)
    states_buy, states_sell, total_gains, invest = agent.buy(initial_money = initial_money)
    #plotting results
    fig = plt.figure(figsize = (15,5))
    plt.plot(df, color='r', lw=2.)
    plt.plot(df, '^', markersize=10, color='m', label = 'buying signal', markevery = states_buy)
    plt.plot(df, 'v', markersize=10, color='k', label = 'selling signal', markevery = states_sell)
    plt.title(f'Buy/sell for {crypto_id} - at the end number of coins: %d, value of coins: %.2f, balance: %.2f, total gain: %.2f' % (len(states_buy) - len(states_sell), round((len(states_buy) - len(states_sell)) * indices['litecoin_Price'][-1], 2), initial_money + total_gains, round((len(states_buy) - len(states_sell)) * indices['litecoin_Price'][-1] + total_gains, 2)))
    plt.legend()
    plt.show()

C:\Users\FP00IF\AppData\Local\Temp\ipykernel_13560\3391229928.py:23: UserWarning:

`tf.layers.dense` is deprecated and will be removed in a future version. Please use `tf.keras.layers.Dense` instead.

C:\Users\FP00IF\AppData\Local\Temp\ipykernel_13560\3391229928.py:24: UserWarning:

`tf.layers.dense` is deprecated and will be removed in a future version. Please use `tf.keras.layers.Dense` instead.

epoch: 10, reward: 1162.227138.3, loss: 0.614986
epoch: 20, reward: 1354.074798.3, loss: 0.260855
epoch: 30, reward: 911.429504.3, loss: 0.162745
epoch: 40, reward: 709.313623.3, loss: 0.127773
epoch: 50, reward: 811.819763.3, loss: 0.110950
epoch: 60, reward: 738.534009.3, loss: 0.100550
epoch: 70, reward: 837.225948.3, loss: 0.092764
epoch: 80, reward: 786.450425.3, loss: 0.086098
epoch: 90, reward: 793.486087.3, loss: 0.080180
epoch: 100, reward: 775.283695.3, loss: 0.075042
day 2: buy 1 unit at price 51.216654, total balance 99948.783346
day 3: buy 1 unit at price 51.216654, total balance 99897.566692
day 5: buy 1 unit at price 51.216654, total balance 99846.350038
day 6, sell 1 unit at price 51.216654, total balance 99897.566692
day 11, sell 1 unit at price 53.469885, total balance 99951.036577
day 12: buy 1 unit at price 52.103387, total balance 99898.933190
day 13, sell 1 unit at price 55.768096, total balance 99954.701286
day 17, sell 1 unit at price 55.910854, total balance 100010.612140
day 18: buy 1 unit at price 52.751012, total balance 99957.861128
day 19, sell 1 unit at price 53.486457, total balance 100011.347585
day 22: buy 1 unit at price 51.216654, total balance 99960.130931
day 24: buy 1 unit at price 52.265822, total balance 99907.865109
day 26, sell 1 unit at price 51.216654, total balance 99959.081763
day 27: buy 1 unit at price 52.203830, total balance 99906.877933
day 28: buy 1 unit at price 51.670780, total balance 99855.207153
day 29: buy 1 unit at price 54.182362, total balance 99801.024791
day 31, sell 1 unit at price 51.216654, total balance 99852.241445
day 32: buy 1 unit at price 51.216654, total balance 99801.024791
day 33, sell 1 unit at price 51.216654, total balance 99852.241445
day 34, sell 1 unit at price 51.242182, total balance 99903.483627
day 37: buy 1 unit at price 55.962800, total balance 99847.520827
day 38, sell 1 unit at price 58.499404, total balance 99906.020231
day 39: buy 1 unit at price 57.942911, total balance 99848.077320
day 41: buy 1 unit at price 58.072482, total balance 99790.004838
day 42: buy 1 unit at price 56.134316, total balance 99733.870523
day 44: buy 1 unit at price 58.468592, total balance 99675.401931
day 47: buy 1 unit at price 58.913699, total balance 99616.488231
day 48, sell 1 unit at price 63.438516, total balance 99679.926747
day 49, sell 1 unit at price 61.098010, total balance 99741.024757
day 50, sell 1 unit at price 60.360575, total balance 99801.385332
day 51: buy 1 unit at price 59.878581, total balance 99741.506751
day 52: buy 1 unit at price 59.463690, total balance 99682.043060
day 54: buy 1 unit at price 57.788245, total balance 99624.254816
day 56, sell 1 unit at price 62.294398, total balance 99686.549213
day 57, sell 1 unit at price 60.993070, total balance 99747.542283
day 58, sell 1 unit at price 61.078303, total balance 99808.620586
day 59: buy 1 unit at price 62.658679, total balance 99745.961907
day 60: buy 1 unit at price 59.304064, total balance 99686.657843
day 61, sell 1 unit at price 61.584801, total balance 99748.242645
day 62: buy 1 unit at price 61.856690, total balance 99686.385955
day 63, sell 1 unit at price 62.996502, total balance 99749.382457
day 65, sell 1 unit at price 63.282892, total balance 99812.665348
day 66: buy 1 unit at price 60.891103, total balance 99751.774246
day 67, sell 1 unit at price 61.394904, total balance 99813.169149
day 69: buy 1 unit at price 60.084630, total balance 99753.084520
day 70: buy 1 unit at price 54.052991, total balance 99699.031529
day 71: buy 1 unit at price 54.097065, total balance 99644.934463
day 74: buy 1 unit at price 57.107544, total balance 99587.826920
day 75, sell 1 unit at price 56.488167, total balance 99644.315087
day 78: buy 1 unit at price 53.085396, total balance 99591.229691
day 81, sell 1 unit at price 53.132720, total balance 99644.362411
day 82: buy 1 unit at price 53.733363, total balance 99590.629049
day 83: buy 1 unit at price 57.575445, total balance 99533.053603
day 86, sell 1 unit at price 60.918783, total balance 99593.972386
day 87: buy 1 unit at price 60.522146, total balance 99533.450240
day 88, sell 1 unit at price 54.221858, total balance 99587.672098
day 89: buy 1 unit at price 57.487712, total balance 99530.184386
day 90: buy 1 unit at price 57.972180, total balance 99472.212206
day 93: buy 1 unit at price 62.152534, total balance 99410.059672
day 95: buy 1 unit at price 59.036000, total balance 99351.023672
day 96, sell 1 unit at price 60.174182, total balance 99411.197854
day 97: buy 1 unit at price 56.212531, total balance 99354.985323
day 98: buy 1 unit at price 55.742103, total balance 99299.243220
day 99: buy 1 unit at price 57.768156, total balance 99241.475064
day 100: buy 1 unit at price 52.888187, total balance 99188.586877
day 102: buy 1 unit at price 52.278880, total balance 99136.307997
day 103, sell 1 unit at price 51.216654, total balance 99187.524651
day 104, sell 1 unit at price 53.586939, total balance 99241.111589
day 105: buy 1 unit at price 55.002573, total balance 99186.109016
day 106: buy 1 unit at price 53.461201, total balance 99132.647815
day 107: buy 1 unit at price 52.582865, total balance 99080.064950
day 108, sell 1 unit at price 53.387989, total balance 99133.452940
day 109: buy 1 unit at price 52.753254, total balance 99080.699685
day 111, sell 1 unit at price 53.959679, total balance 99134.659364
day 114: buy 1 unit at price 51.897575, total balance 99082.761790
day 117: buy 1 unit at price 54.581740, total balance 99028.180049
day 118: buy 1 unit at price 53.532409, total balance 98974.647640
day 121, sell 1 unit at price 53.906706, total balance 99028.554346
day 123: buy 1 unit at price 52.027246, total balance 98976.527099
day 125: buy 1 unit at price 51.217335, total balance 98925.309764
day 126: buy 1 unit at price 51.299957, total balance 98874.009808
day 128, sell 1 unit at price 51.487928, total balance 98925.497736
day 129, sell 1 unit at price 51.918242, total balance 98977.415978
day 130, sell 1 unit at price 51.932672, total balance 99029.348650
day 131: buy 1 unit at price 51.216654, total balance 98978.131996
day 132: buy 1 unit at price 51.216654, total balance 98926.915342
day 134: buy 1 unit at price 52.162973, total balance 98874.752369
day 135: buy 1 unit at price 53.913665, total balance 98820.838704
day 137, sell 1 unit at price 56.117468, total balance 98876.956173
day 138: buy 1 unit at price 56.395931, total balance 98820.560241
day 139, sell 1 unit at price 54.816594, total balance 98875.376835
day 140, sell 1 unit at price 54.989780, total balance 98930.366615
day 141, sell 1 unit at price 56.448468, total balance 98986.815083
day 143, sell 1 unit at price 55.074291, total balance 99041.889374
day 144, sell 1 unit at price 55.135495, total balance 99097.024869
day 147, sell 1 unit at price 67.564797, total balance 99164.589666
day 148: buy 1 unit at price 69.655678, total balance 99094.933988
day 149, sell 1 unit at price 67.939525, total balance 99162.873514
day 150: buy 1 unit at price 67.728240, total balance 99095.145274
day 151, sell 1 unit at price 57.346729, total balance 99152.492002
day 152: buy 1 unit at price 51.216654, total balance 99101.275348
day 153: buy 1 unit at price 60.420301, total balance 99040.855047
day 156: buy 1 unit at price 57.452704, total balance 98983.402343
day 157: buy 1 unit at price 56.595295, total balance 98926.807048
day 159, sell 1 unit at price 57.573838, total balance 98984.380886
day 160: buy 1 unit at price 62.421922, total balance 98921.958964
day 162, sell 1 unit at price 63.832266, total balance 98985.791230
day 163: buy 1 unit at price 62.029396, total balance 98923.761834
day 164, sell 1 unit at price 61.400869, total balance 98985.162703
day 165: buy 1 unit at price 70.339749, total balance 98914.822954
day 166, sell 1 unit at price 79.217674, total balance 98994.040628
day 167, sell 1 unit at price 78.822501, total balance 99072.863129
day 168: buy 1 unit at price 74.035992, total balance 98998.827137
day 169: buy 1 unit at price 76.342855, total balance 98922.484282
day 170: buy 1 unit at price 75.012093, total balance 98847.472190
day 172, sell 1 unit at price 75.983522, total balance 98923.455712
day 173, sell 1 unit at price 79.343687, total balance 99002.799400
day 175, sell 1 unit at price 78.178622, total balance 99080.978022
day 177: buy 1 unit at price 77.324529, total balance 99003.653493
day 178: buy 1 unit at price 80.256749, total balance 98923.396744
day 179, sell 1 unit at price 79.974647, total balance 99003.371391
day 180: buy 1 unit at price 77.018333, total balance 98926.353058
day 181: buy 1 unit at price 78.506998, total balance 98847.846061
day 183, sell 1 unit at price 76.336368, total balance 98924.182429
day 184: buy 1 unit at price 76.603671, total balance 98847.578758
day 185, sell 1 unit at price 76.203646, total balance 98923.782403
day 188: buy 1 unit at price 72.161022, total balance 98851.621382
day 189: buy 1 unit at price 66.156627, total balance 98785.464755
day 190, sell 1 unit at price 65.053041, total balance 98850.517796
day 191: buy 1 unit at price 64.412793, total balance 98786.105002
day 192: buy 1 unit at price 63.334417, total balance 98722.770586
day 193: buy 1 unit at price 65.698434, total balance 98657.072152
day 196, sell 1 unit at price 65.779472, total balance 98722.851623
day 197, sell 1 unit at price 65.809458, total balance 98788.661081
day 200: buy 1 unit at price 68.722001, total balance 98719.939080
day 201: buy 1 unit at price 66.276459, total balance 98653.662621
day 204, sell 1 unit at price 69.946563, total balance 98723.609184
day 205: buy 1 unit at price 70.762545, total balance 98652.846639
day 206, sell 1 unit at price 74.805526, total balance 98727.652165
day 208, sell 1 unit at price 75.503200, total balance 98803.155366
day 209: buy 1 unit at price 74.160929, total balance 98728.994436
day 210, sell 1 unit at price 75.986453, total balance 98804.980890
day 212, sell 1 unit at price 77.511191, total balance 98882.492081
day 213: buy 1 unit at price 81.767648, total balance 98800.724433
day 216, sell 1 unit at price 85.940197, total balance 98886.664630
day 217, sell 1 unit at price 86.439572, total balance 98973.104201
day 218, sell 1 unit at price 87.595730, total balance 99060.699931
day 219, sell 1 unit at price 87.262728, total balance 99147.962659
day 220: buy 1 unit at price 85.779206, total balance 99062.183453
day 221: buy 1 unit at price 87.195860, total balance 98974.987593
day 222, sell 1 unit at price 82.865054, total balance 99057.852647
day 223: buy 1 unit at price 84.179015, total balance 98973.673632
day 224: buy 1 unit at price 90.612303, total balance 98883.061329
day 225, sell 1 unit at price 87.979159, total balance 98971.040488
day 226, sell 1 unit at price 87.660317, total balance 99058.700805
day 227, sell 1 unit at price 90.187979, total balance 99148.888784
day 228: buy 1 unit at price 87.189962, total balance 99061.698822
day 229, sell 1 unit at price 89.619908, total balance 99151.318729
day 230, sell 1 unit at price 87.530363, total balance 99238.849092
day 231: buy 1 unit at price 88.868573, total balance 99149.980519
day 232: buy 1 unit at price 89.658204, total balance 99060.322315
day 233, sell 1 unit at price 95.046393, total balance 99155.368708
day 235, sell 1 unit at price 94.403882, total balance 99249.772590
day 236: buy 1 unit at price 96.075682, total balance 99153.696907
day 237: buy 1 unit at price 96.075682, total balance 99057.621225
day 238, sell 1 unit at price 96.075682, total balance 99153.696907
day 239: buy 1 unit at price 96.075682, total balance 99057.621225
day 240: buy 1 unit at price 96.075682, total balance 98961.545543
day 241, sell 1 unit at price 96.075682, total balance 99057.621225
day 242: buy 1 unit at price 96.075682, total balance 98961.545543
day 243, sell 1 unit at price 96.075682, total balance 99057.621225
day 244: buy 1 unit at price 92.117770, total balance 98965.503455
day 246: buy 1 unit at price 94.513574, total balance 98870.989881
day 247: buy 1 unit at price 92.315673, total balance 98778.674207
day 248: buy 1 unit at price 91.210468, total balance 98687.463740
day 249, sell 1 unit at price 95.645630, total balance 98783.109370
day 251, sell 1 unit at price 96.075682, total balance 98879.185052
day 252, sell 1 unit at price 96.075682, total balance 98975.260735
day 253: buy 1 unit at price 96.075682, total balance 98879.185052
day 255, sell 1 unit at price 95.911047, total balance 98975.096099
day 256, sell 1 unit at price 93.694908, total balance 99068.791008
day 258, sell 1 unit at price 94.763655, total balance 99163.554663
day 259: buy 1 unit at price 91.528892, total balance 99072.025770
day 260, sell 1 unit at price 93.169766, total balance 99165.195537
day 263, sell 1 unit at price 93.730469, total balance 99258.926006
day 264: buy 1 unit at price 96.075682, total balance 99162.850324
day 265: buy 1 unit at price 95.277903, total balance 99067.572421
day 266, sell 1 unit at price 90.576350, total balance 99158.148771
day 268: buy 1 unit at price 89.981976, total balance 99068.166794
day 269: buy 1 unit at price 87.410852, total balance 98980.755942
day 270, sell 1 unit at price 86.361797, total balance 99067.117740
day 271: buy 1 unit at price 82.834080, total balance 98984.283660
day 272: buy 1 unit at price 76.706116, total balance 98907.577544
day 273: buy 1 unit at price 71.761925, total balance 98835.815618
day 274, sell 1 unit at price 69.294091, total balance 98905.109710
day 277, sell 1 unit at price 84.336405, total balance 98989.446115
day 279: buy 1 unit at price 79.427833, total balance 98910.018282
day 280: buy 1 unit at price 85.604171, total balance 98824.414110
day 283: buy 1 unit at price 78.645758, total balance 98745.768352
day 284: buy 1 unit at price 82.978275, total balance 98662.790077
day 285: buy 1 unit at price 87.862327, total balance 98574.927750
day 288, sell 1 unit at price 92.040299, total balance 98666.968049
day 290: buy 1 unit at price 89.428626, total balance 98577.539423
day 293: buy 1 unit at price 89.115241, total balance 98488.424182
day 294, sell 1 unit at price 89.710197, total balance 98578.134379
day 295: buy 1 unit at price 92.681512, total balance 98485.452867
day 298, sell 1 unit at price 92.531440, total balance 98577.984307
day 299: buy 1 unit at price 92.686518, total balance 98485.297789
day 300, sell 1 unit at price 90.920143, total balance 98576.217932
day 302: buy 1 unit at price 90.002259, total balance 98486.215673
day 304, sell 1 unit at price 93.833757, total balance 98580.049430
day 307: buy 1 unit at price 94.245149, total balance 98485.804281
day 308: buy 1 unit at price 96.075682, total balance 98389.728598
day 311, sell 1 unit at price 96.075682, total balance 98485.804281
day 312: buy 1 unit at price 96.075682, total balance 98389.728598
day 314, sell 1 unit at price 90.234103, total balance 98479.962702
day 315: buy 1 unit at price 85.478007, total balance 98394.484695
day 316, sell 1 unit at price 87.156280, total balance 98481.640975
day 317: buy 1 unit at price 86.879429, total balance 98394.761546
day 318: buy 1 unit at price 88.374903, total balance 98306.386643
day 319: buy 1 unit at price 91.232688, total balance 98215.153955
day 320, sell 1 unit at price 88.103764, total balance 98303.257719
day 321, sell 1 unit at price 89.705820, total balance 98392.963539
day 322: buy 1 unit at price 89.707686, total balance 98303.255854
day 324, sell 1 unit at price 88.723579, total balance 98391.979432
day 325: buy 1 unit at price 86.760258, total balance 98305.219174
day 326: buy 1 unit at price 88.314827, total balance 98216.904347
day 328, sell 1 unit at price 88.492922, total balance 98305.397269
day 329, sell 1 unit at price 88.701989, total balance 98394.099258
day 330: buy 1 unit at price 83.487024, total balance 98310.612234
day 331: buy 1 unit at price 83.823306, total balance 98226.788928
day 332: buy 1 unit at price 77.880957, total balance 98148.907971
day 333: buy 1 unit at price 79.906535, total balance 98069.001436
day 334: buy 1 unit at price 81.141020, total balance 97987.860416
day 335, sell 1 unit at price 81.147844, total balance 98069.008261
day 336: buy 1 unit at price 80.566645, total balance 97988.441616
day 341, sell 1 unit at price 93.782631, total balance 98082.224247
day 343, sell 1 unit at price 91.750881, total balance 98173.975128
day 344: buy 1 unit at price 92.297024, total balance 98081.678104
day 345, sell 1 unit at price 92.344455, total balance 98174.022559
day 346: buy 1 unit at price 90.844322, total balance 98083.178237
day 348: buy 1 unit at price 85.992431, total balance 97997.185806
day 349: buy 1 unit at price 86.521451, total balance 97910.664355
day 352, sell 1 unit at price 91.556588, total balance 98002.220943
day 353, sell 1 unit at price 90.960515, total balance 98093.181458
day 358, sell 1 unit at price 95.922770, total balance 98189.104228

epoch: 10, reward: 0.003137.3, loss: 0.000000
epoch: 20, reward: 0.003173.3, loss: 0.000000
epoch: 30, reward: -0.006061.3, loss: 0.000000
epoch: 40, reward: -0.000065.3, loss: 0.000000
epoch: 50, reward: 0.003463.3, loss: 0.000000
epoch: 60, reward: 0.006831.3, loss: 0.000000
epoch: 70, reward: 0.005731.3, loss: 0.000000
epoch: 80, reward: 0.008738.3, loss: 0.000000
epoch: 90, reward: 0.010124.3, loss: 0.000000
epoch: 100, reward: 0.006507.3, loss: 0.000000
day 1: buy 1 unit at price 0.027636, total balance 99999.972364
day 2: buy 1 unit at price 0.025897, total balance 99999.946467
day 3: buy 1 unit at price 0.023805, total balance 99999.922662
day 4: buy 1 unit at price 0.024064, total balance 99999.898598
day 5: buy 1 unit at price 0.025338, total balance 99999.873259
day 6: buy 1 unit at price 0.022721, total balance 99999.850538
day 7, sell 1 unit at price 0.023421, total balance 99999.873959
day 8, sell 1 unit at price 0.021766, total balance 99999.895725
day 9: buy 1 unit at price 0.022959, total balance 99999.872767
day 10, sell 1 unit at price 0.023647, total balance 99999.896414
day 11: buy 1 unit at price 0.023635, total balance 99999.872779
day 12, sell 1 unit at price 0.022844, total balance 99999.895623
day 17, sell 1 unit at price 0.024651, total balance 99999.920274
day 18: buy 1 unit at price 0.023452, total balance 99999.896822
day 19, sell 1 unit at price 0.022709, total balance 99999.919531
day 20, sell 1 unit at price 0.022432, total balance 99999.941963
day 21, sell 1 unit at price 0.022363, total balance 99999.964326
day 22: buy 1 unit at price 0.022392, total balance 99999.941934
day 23, sell 1 unit at price 0.022600, total balance 99999.964534
day 24: buy 1 unit at price 0.023339, total balance 99999.941195
day 25, sell 1 unit at price 0.022740, total balance 99999.963935
day 26, sell 1 unit at price 0.023679, total balance 99999.987614
day 34: buy 1 unit at price 0.022810, total balance 99999.964805
day 35, sell 1 unit at price 0.023000, total balance 99999.987805
day 37: buy 1 unit at price 0.023290, total balance 99999.964515
day 40, sell 1 unit at price 0.025394, total balance 99999.989909
day 44: buy 1 unit at price 0.025129, total balance 99999.964780
day 45, sell 1 unit at price 0.023553, total balance 99999.988333
day 46: buy 1 unit at price 0.023329, total balance 99999.965004
day 47, sell 1 unit at price 0.025097, total balance 99999.990102
day 61: buy 1 unit at price 0.029451, total balance 99999.960650
day 62, sell 1 unit at price 0.029451, total balance 99999.990102
day 71: buy 1 unit at price 0.025829, total balance 99999.964273
day 74, sell 1 unit at price 0.026524, total balance 99999.990797
day 86: buy 1 unit at price 0.024992, total balance 99999.965806
day 87, sell 1 unit at price 0.024458, total balance 99999.990263
day 93: buy 1 unit at price 0.025660, total balance 99999.964603
day 94, sell 1 unit at price 0.025622, total balance 99999.990225
day 96: buy 1 unit at price 0.024063, total balance 99999.966162
day 99, sell 1 unit at price 0.024370, total balance 99999.990533
day 100: buy 1 unit at price 0.022470, total balance 99999.968062
day 101, sell 1 unit at price 0.022909, total balance 99999.990971
day 103: buy 1 unit at price 0.022041, total balance 99999.968930
day 104, sell 1 unit at price 0.022956, total balance 99999.991886
day 109: buy 1 unit at price 0.022705, total balance 99999.969181
day 110, sell 1 unit at price 0.022950, total balance 99999.992131
day 121: buy 1 unit at price 0.023121, total balance 99999.969010
day 122: buy 1 unit at price 0.022808, total balance 99999.946202
day 123: buy 1 unit at price 0.022734, total balance 99999.923468
day 124, sell 1 unit at price 0.022811, total balance 99999.946278
day 125, sell 1 unit at price 0.023038, total balance 99999.969317
day 126: buy 1 unit at price 0.022836, total balance 99999.946480
day 127, sell 1 unit at price 0.022800, total balance 99999.969281
day 128: buy 1 unit at price 0.022929, total balance 99999.946352
day 129, sell 1 unit at price 0.023352, total balance 99999.969704
day 131, sell 1 unit at price 0.022342, total balance 99999.992046
day 135: buy 1 unit at price 0.022818, total balance 99999.969228
day 136, sell 1 unit at price 0.022431, total balance 99999.991659
day 140: buy 1 unit at price 0.023529, total balance 99999.968131
day 141, sell 1 unit at price 0.023963, total balance 99999.992094
day 145: buy 1 unit at price 0.023182, total balance 99999.968912
day 146, sell 1 unit at price 0.023706, total balance 99999.992618
day 152: buy 1 unit at price 0.019579, total balance 99999.973039
day 153, sell 1 unit at price 0.022318, total balance 99999.995357
day 154: buy 1 unit at price 0.021516, total balance 99999.973842
day 155, sell 1 unit at price 0.020159, total balance 99999.994001
day 157: buy 1 unit at price 0.019167, total balance 99999.974834
day 158, sell 1 unit at price 0.019763, total balance 99999.994597
day 160: buy 1 unit at price 0.019393, total balance 99999.975204
day 161: buy 1 unit at price 0.019279, total balance 99999.955925
day 162, sell 1 unit at price 0.019316, total balance 99999.975241
day 163, sell 1 unit at price 0.018105, total balance 99999.993347
day 167: buy 1 unit at price 0.019170, total balance 99999.974177
day 168, sell 1 unit at price 0.019021, total balance 99999.993198
day 170: buy 1 unit at price 0.018810, total balance 99999.974388
day 172, sell 1 unit at price 0.018553, total balance 99999.992940
day 173: buy 1 unit at price 0.019323, total balance 99999.973617
day 174, sell 1 unit at price 0.019083, total balance 99999.992701
day 177: buy 1 unit at price 0.019364, total balance 99999.973336
day 178, sell 1 unit at price 0.019219, total balance 99999.992555
day 186: buy 1 unit at price 0.019078, total balance 99999.973477
day 187, sell 1 unit at price 0.018744, total balance 99999.992221
day 189: buy 1 unit at price 0.016939, total balance 99999.975282
day 190: buy 1 unit at price 0.016899, total balance 99999.958383
day 192: buy 1 unit at price 0.016402, total balance 99999.941982
day 193, sell 1 unit at price 0.016402, total balance 99999.958383
day 194: buy 1 unit at price 0.016402, total balance 99999.941982
day 195, sell 1 unit at price 0.016402, total balance 99999.958383
day 197: buy 1 unit at price 0.016402, total balance 99999.941982
day 198: buy 1 unit at price 0.016402, total balance 99999.925580
day 199, sell 1 unit at price 0.016402, total balance 99999.941982
day 200, sell 1 unit at price 0.016402, total balance 99999.958383
day 201: buy 1 unit at price 0.016402, total balance 99999.941982
day 202: buy 1 unit at price 0.016402, total balance 99999.925580
day 203: buy 1 unit at price 0.016402, total balance 99999.909179
day 204: buy 1 unit at price 0.016402, total balance 99999.892777
day 205: buy 1 unit at price 0.016402, total balance 99999.876376
day 206: buy 1 unit at price 0.016402, total balance 99999.859974
day 207, sell 1 unit at price 0.016402, total balance 99999.876376
day 208, sell 1 unit at price 0.016589, total balance 99999.892964
day 209, sell 1 unit at price 0.016402, total balance 99999.909366
day 210, sell 1 unit at price 0.016443, total balance 99999.925809
day 211, sell 1 unit at price 0.016402, total balance 99999.942210
day 212, sell 1 unit at price 0.017053, total balance 99999.959263
day 213, sell 1 unit at price 0.017518, total balance 99999.976781
day 214, sell 1 unit at price 0.018150, total balance 99999.994931
day 237: buy 1 unit at price 0.024167, total balance 99999.970764
day 239, sell 1 unit at price 0.024953, total balance 99999.995717
day 244: buy 1 unit at price 0.023270, total balance 99999.972447
day 245, sell 1 unit at price 0.023628, total balance 99999.996075
day 248: buy 1 unit at price 0.023039, total balance 99999.973036
day 249: buy 1 unit at price 0.023888, total balance 99999.949148
day 250, sell 1 unit at price 0.025636, total balance 99999.974784
day 251, sell 1 unit at price 0.024539, total balance 99999.999323
day 256: buy 1 unit at price 0.029451, total balance 99999.969872
day 257, sell 1 unit at price 0.029451, total balance 99999.999323
day 260: buy 1 unit at price 0.026963, total balance 99999.972361
day 261, sell 1 unit at price 0.028206, total balance 100000.000567
day 263: buy 1 unit at price 0.026964, total balance 99999.973602
day 264, sell 1 unit at price 0.028116, total balance 100000.001719
day 269: buy 1 unit at price 0.024648, total balance 99999.977070
day 270, sell 1 unit at price 0.023617, total balance 100000.000687
day 274: buy 1 unit at price 0.020385, total balance 99999.980302
day 275: buy 1 unit at price 0.022048, total balance 99999.958254
day 276, sell 1 unit at price 0.023554, total balance 99999.981807
day 277, sell 1 unit at price 0.023990, total balance 100000.005798
day 281: buy 1 unit at price 0.023131, total balance 99999.982667
day 282, sell 1 unit at price 0.024703, total balance 100000.007369
day 299: buy 1 unit at price 0.024371, total balance 99999.982999
day 301, sell 1 unit at price 0.024173, total balance 100000.007171
day 304: buy 1 unit at price 0.024969, total balance 99999.982202
day 306, sell 1 unit at price 0.024477, total balance 100000.006678
day 309: buy 1 unit at price 0.025731, total balance 99999.980947
day 310, sell 1 unit at price 0.026193, total balance 100000.007140
day 311: buy 1 unit at price 0.025480, total balance 99999.981661
day 312, sell 1 unit at price 0.026448, total balance 100000.008108
day 315: buy 1 unit at price 0.022162, total balance 99999.985946
day 316: buy 1 unit at price 0.022721, total balance 99999.963225
day 318: buy 1 unit at price 0.022106, total balance 99999.941118
day 319, sell 1 unit at price 0.022719, total balance 99999.963838
day 320, sell 1 unit at price 0.022011, total balance 99999.985849
day 322, sell 1 unit at price 0.022437, total balance 100000.008286
day 323: buy 1 unit at price 0.022670, total balance 99999.985616
day 324, sell 1 unit at price 0.022320, total balance 100000.007936
day 327: buy 1 unit at price 0.020935, total balance 99999.987001
day 328, sell 1 unit at price 0.020785, total balance 100000.007786
day 329: buy 1 unit at price 0.021186, total balance 99999.986600
day 330, sell 1 unit at price 0.020337, total balance 100000.006937
day 332: buy 1 unit at price 0.019360, total balance 99999.987578
day 333, sell 1 unit at price 0.019387, total balance 100000.006965
day 339: buy 1 unit at price 0.019499, total balance 99999.987465
day 340, sell 1 unit at price 0.019477, total balance 100000.006943
day 341: buy 1 unit at price 0.019962, total balance 99999.986981
day 343: buy 1 unit at price 0.019579, total balance 99999.967402
day 344, sell 1 unit at price 0.019415, total balance 99999.986817
day 346: buy 1 unit at price 0.019334, total balance 99999.967483
day 347, sell 1 unit at price 0.019408, total balance 99999.986891
day 348: buy 1 unit at price 0.019487, total balance 99999.967404
day 349, sell 1 unit at price 0.019285, total balance 99999.986689
day 350, sell 1 unit at price 0.019279, total balance 100000.005968

epoch: 10, reward: -31.843137.3, loss: 0.001820
epoch: 20, reward: -36.280919.3, loss: 0.001698
epoch: 30, reward: -29.514510.3, loss: 0.001617
epoch: 40, reward: -29.429248.3, loss: 0.001583
epoch: 50, reward: -28.068003.3, loss: 0.001473
epoch: 60, reward: -22.629463.3, loss: 0.001408
epoch: 70, reward: -29.504909.3, loss: 0.001465
epoch: 80, reward: -29.377293.3, loss: 0.001279
epoch: 90, reward: -29.377293.3, loss: 0.001228
epoch: 100, reward: -26.475258.3, loss: 0.001170
day 1: buy 1 unit at price 6.094975, total balance 99993.905025
day 3: buy 1 unit at price 5.335227, total balance 99988.569798
day 5, sell 1 unit at price 5.836534, total balance 99994.406333
day 6: buy 1 unit at price 5.290282, total balance 99989.116050
day 7: buy 1 unit at price 5.349333, total balance 99983.766717
day 8: buy 1 unit at price 5.115148, total balance 99978.651569
day 10: buy 1 unit at price 5.475020, total balance 99973.176549
day 12: buy 1 unit at price 5.536857, total balance 99967.639692
day 14, sell 1 unit at price 6.178812, total balance 99973.818504
day 15: buy 1 unit at price 6.085506, total balance 99967.732999
day 16, sell 1 unit at price 5.849084, total balance 99973.582082
day 19: buy 1 unit at price 5.447204, total balance 99968.134878
day 21: buy 1 unit at price 5.334754, total balance 99962.800124
day 22, sell 1 unit at price 5.344213, total balance 99968.144337
day 25: buy 1 unit at price 5.493144, total balance 99962.651193
day 26: buy 1 unit at price 5.578693, total balance 99957.072500
day 27, sell 1 unit at price 5.816426, total balance 99962.888926
day 28, sell 1 unit at price 5.780970, total balance 99968.669896
day 29: buy 1 unit at price 5.804425, total balance 99962.865471
day 33, sell 1 unit at price 5.296906, total balance 99968.162378
day 34: buy 1 unit at price 5.361619, total balance 99962.800759
day 35: buy 1 unit at price 5.224991, total balance 99957.575768
day 36: buy 1 unit at price 5.395054, total balance 99952.180714
day 39: buy 1 unit at price 6.056970, total balance 99946.123743
day 40: buy 1 unit at price 5.579303, total balance 99940.544440
day 42: buy 1 unit at price 5.482182, total balance 99935.062258
day 43, sell 1 unit at price 5.592758, total balance 99940.655016
day 44: buy 1 unit at price 5.655278, total balance 99934.999737
day 45: buy 1 unit at price 5.252125, total balance 99929.747612
day 46: buy 1 unit at price 5.305269, total balance 99924.442343
day 49: buy 1 unit at price 5.977898, total balance 99918.464445
day 53: buy 1 unit at price 8.108303, total balance 99910.356142
day 55: buy 1 unit at price 8.104364, total balance 99902.251778
day 57, sell 1 unit at price 8.108303, total balance 99910.360081
day 59: buy 1 unit at price 8.108303, total balance 99902.251778
day 62: buy 1 unit at price 8.108303, total balance 99894.143474
day 63: buy 1 unit at price 8.108303, total balance 99886.035171
day 66, sell 1 unit at price 8.108303, total balance 99894.143474
day 70: buy 1 unit at price 6.487358, total balance 99887.656116
day 74, sell 1 unit at price 6.386694, total balance 99894.042810
day 75: buy 1 unit at price 6.279919, total balance 99887.762891
day 77: buy 1 unit at price 5.637781, total balance 99882.125110
day 78: buy 1 unit at price 5.846626, total balance 99876.278484
day 79: buy 1 unit at price 5.652479, total balance 99870.626006
day 81: buy 1 unit at price 5.710401, total balance 99864.915605
day 82: buy 1 unit at price 5.732290, total balance 99859.183314
day 83: buy 1 unit at price 5.775727, total balance 99853.407588
day 84: buy 1 unit at price 5.820387, total balance 99847.587201
day 85, sell 1 unit at price 6.512140, total balance 99854.099341
day 86: buy 1 unit at price 6.266124, total balance 99847.833217
day 87, sell 1 unit at price 6.135254, total balance 99853.968471
day 88: buy 1 unit at price 5.612293, total balance 99848.356178
day 90: buy 1 unit at price 5.867763, total balance 99842.488415
day 91, sell 1 unit at price 6.375064, total balance 99848.863478
day 92, sell 1 unit at price 6.365403, total balance 99855.228882
day 93: buy 1 unit at price 6.395707, total balance 99848.833175
day 95: buy 1 unit at price 5.867238, total balance 99842.965937
day 96, sell 1 unit at price 5.974583, total balance 99848.940520
day 97: buy 1 unit at price 5.717181, total balance 99843.223339
day 100: buy 1 unit at price 5.256584, total balance 99837.966755
day 101: buy 1 unit at price 5.487971, total balance 99832.478784
day 103: buy 1 unit at price 5.275511, total balance 99827.203273
day 104, sell 1 unit at price 5.536144, total balance 99832.739416
day 108: buy 1 unit at price 5.680937, total balance 99827.058479
day 109: buy 1 unit at price 5.734628, total balance 99821.323850
day 110: buy 1 unit at price 5.868534, total balance 99815.455317
day 111: buy 1 unit at price 5.774667, total balance 99809.680650
day 113, sell 1 unit at price 5.519499, total balance 99815.200149
day 118: buy 1 unit at price 5.485733, total balance 99809.714416
day 120: buy 1 unit at price 5.399329, total balance 99804.315087
day 121, sell 1 unit at price 5.427059, total balance 99809.742146
day 122: buy 1 unit at price 5.157920, total balance 99804.584226
day 127: buy 1 unit at price 5.192161, total balance 99799.392064
day 128: buy 1 unit at price 5.151015, total balance 99794.241050
day 130: buy 1 unit at price 5.133745, total balance 99789.107305
day 132: buy 1 unit at price 4.941263, total balance 99784.166042
day 134, sell 1 unit at price 4.950340, total balance 99789.116381
day 135, sell 1 unit at price 5.110065, total balance 99794.226446
day 136: buy 1 unit at price 4.945212, total balance 99789.281235
day 139, sell 1 unit at price 5.238283, total balance 99794.519518
day 140: buy 1 unit at price 5.371069, total balance 99789.148449
day 143, sell 1 unit at price 5.439023, total balance 99794.587472
day 144, sell 1 unit at price 5.371205, total balance 99799.958677
day 146: buy 1 unit at price 5.782668, total balance 99794.176009
day 149: buy 1 unit at price 5.841506, total balance 99788.334503
day 153, sell 1 unit at price 4.639939, total balance 99792.974442
day 155, sell 1 unit at price 4.256944, total balance 99797.231386
day 156, sell 1 unit at price 4.303318, total balance 99801.534704
day 158: buy 1 unit at price 4.392239, total balance 99797.142465
day 159: buy 1 unit at price 4.344666, total balance 99792.797799
day 160, sell 1 unit at price 4.323592, total balance 99797.121390
day 161: buy 1 unit at price 4.344691, total balance 99792.776700
day 163: buy 1 unit at price 4.168899, total balance 99788.607801
day 164: buy 1 unit at price 4.168274, total balance 99784.439526
day 165: buy 1 unit at price 4.320693, total balance 99780.118833
day 166: buy 1 unit at price 4.444423, total balance 99775.674410
day 167: buy 1 unit at price 4.393356, total balance 99771.281054
day 175: buy 1 unit at price 4.657283, total balance 99766.623771
day 176, sell 1 unit at price 4.424802, total balance 99771.048573
day 177, sell 1 unit at price 4.576935, total balance 99775.625507
day 178: buy 1 unit at price 4.522686, total balance 99771.102822
day 182: buy 1 unit at price 4.439656, total balance 99766.663166
day 184, sell 1 unit at price 4.292581, total balance 99770.955747
day 186: buy 1 unit at price 4.316966, total balance 99766.638782
day 187: buy 1 unit at price 4.230005, total balance 99762.408776
day 188, sell 1 unit at price 4.142220, total balance 99766.550997
day 189: buy 1 unit at price 3.221796, total balance 99763.329200
day 191: buy 1 unit at price 3.221796, total balance 99760.107404
day 193, sell 1 unit at price 3.221796, total balance 99763.329200
day 194: buy 1 unit at price 3.221796, total balance 99760.107404
day 195: buy 1 unit at price 3.221796, total balance 99756.885607
day 197, sell 1 unit at price 3.221796, total balance 99760.107404
day 198: buy 1 unit at price 3.221796, total balance 99756.885607
day 199: buy 1 unit at price 3.221796, total balance 99753.663811
day 201, sell 1 unit at price 3.221796, total balance 99756.885607
day 202: buy 1 unit at price 3.221796, total balance 99753.663811
day 203: buy 1 unit at price 3.221796, total balance 99750.442015
day 204: buy 1 unit at price 3.221796, total balance 99747.220218
day 207: buy 1 unit at price 3.221796, total balance 99743.998422
day 210, sell 1 unit at price 3.342226, total balance 99747.340647
day 212: buy 1 unit at price 3.424543, total balance 99743.916105
day 213: buy 1 unit at price 3.788294, total balance 99740.127810
day 214, sell 1 unit at price 3.754659, total balance 99743.882469
day 215, sell 1 unit at price 3.728150, total balance 99747.610620
day 216: buy 1 unit at price 3.849902, total balance 99743.760718
day 219: buy 1 unit at price 4.705816, total balance 99739.054902
day 225, sell 1 unit at price 4.942067, total balance 99743.996969
day 227: buy 1 unit at price 5.436188, total balance 99738.560781
day 228: buy 1 unit at price 4.969106, total balance 99733.591676
day 232, sell 1 unit at price 5.274458, total balance 99738.866134
day 236, sell 1 unit at price 5.688103, total balance 99744.554238
day 238: buy 1 unit at price 5.685113, total balance 99738.869125
day 244: buy 1 unit at price 4.777709, total balance 99734.091416
day 245, sell 1 unit at price 4.858238, total balance 99738.949654
day 247: buy 1 unit at price 4.909973, total balance 99734.039681
day 248, sell 1 unit at price 4.866140, total balance 99738.905821
day 249, sell 1 unit at price 5.085197, total balance 99743.991018
day 251: buy 1 unit at price 5.428213, total balance 99738.562805
day 253: buy 1 unit at price 7.501269, total balance 99731.061537
day 255: buy 1 unit at price 8.108303, total balance 99722.953233
day 259, sell 1 unit at price 6.844829, total balance 99729.798063
day 261, sell 1 unit at price 6.877516, total balance 99736.675579
day 263: buy 1 unit at price 6.477690, total balance 99730.197889
day 264, sell 1 unit at price 7.217167, total balance 99737.415056
day 266: buy 1 unit at price 6.248119, total balance 99731.166937
day 267: buy 1 unit at price 5.914069, total balance 99725.252868
day 271: buy 1 unit at price 5.799898, total balance 99719.452970
day 272: buy 1 unit at price 5.503809, total balance 99713.949161
day 273: buy 1 unit at price 5.318301, total balance 99708.630860
day 277: buy 1 unit at price 6.776869, total balance 99701.853991
day 278: buy 1 unit at price 5.616347, total balance 99696.237644
day 280: buy 1 unit at price 6.425595, total balance 99689.812049
day 281: buy 1 unit at price 5.903686, total balance 99683.908363
day 282, sell 1 unit at price 6.333011, total balance 99690.241374
day 283: buy 1 unit at price 5.747303, total balance 99684.494071
day 284, sell 1 unit at price 5.962963, total balance 99690.457034
day 287: buy 1 unit at price 5.558530, total balance 99684.898504
day 289: buy 1 unit at price 5.702538, total balance 99679.195965
day 292: buy 1 unit at price 5.740214, total balance 99673.455752
day 293: buy 1 unit at price 5.564597, total balance 99667.891154
day 296: buy 1 unit at price 5.560858, total balance 99662.330296
day 297: buy 1 unit at price 5.575976, total balance 99656.754320
day 300: buy 1 unit at price 5.601345, total balance 99651.152976
day 301: buy 1 unit at price 5.800706, total balance 99645.352269
day 304, sell 1 unit at price 6.176612, total balance 99651.528881
day 305: buy 1 unit at price 6.115595, total balance 99645.413286
day 306: buy 1 unit at price 5.965189, total balance 99639.448097
day 308, sell 1 unit at price 6.225624, total balance 99645.673721
day 310: buy 1 unit at price 6.336287, total balance 99639.337434
day 311: buy 1 unit at price 6.125859, total balance 99633.211575
day 313: buy 1 unit at price 5.697440, total balance 99627.514135
day 315: buy 1 unit at price 5.254582, total balance 99622.259554
day 318: buy 1 unit at price 5.254874, total balance 99617.004680
day 320: buy 1 unit at price 5.255086, total balance 99611.749594
day 321: buy 1 unit at price 5.396816, total balance 99606.352778
day 323: buy 1 unit at price 5.569701, total balance 99600.783077
day 324: buy 1 unit at price 5.450007, total balance 99595.333069
day 325: buy 1 unit at price 5.306684, total balance 99590.026385
day 327: buy 1 unit at price 5.346516, total balance 99584.679869
day 328: buy 1 unit at price 5.263578, total balance 99579.416291
day 329: buy 1 unit at price 5.271028, total balance 99574.145263
day 330, sell 1 unit at price 5.059766, total balance 99579.205029
day 332: buy 1 unit at price 4.543673, total balance 99574.661357
day 336, sell 1 unit at price 4.434751, total balance 99579.096108
day 337: buy 1 unit at price 4.386615, total balance 99574.709493
day 338, sell 1 unit at price 4.448018, total balance 99579.157511
day 339: buy 1 unit at price 4.471232, total balance 99574.686279
day 340: buy 1 unit at price 4.424496, total balance 99570.261783
day 341: buy 1 unit at price 4.545559, total balance 99565.716224
day 342: buy 1 unit at price 4.458812, total balance 99561.257412
day 343: buy 1 unit at price 4.494703, total balance 99556.762709
day 345, sell 1 unit at price 4.382820, total balance 99561.145529
day 346: buy 1 unit at price 4.414707, total balance 99556.730822
day 348: buy 1 unit at price 4.463862, total balance 99552.266959
day 353, sell 1 unit at price 4.635074, total balance 99556.902033
day 357, sell 1 unit at price 4.966965, total balance 99561.868998